{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>short_description</th>\n",
       "      <th>headline</th>\n",
       "      <th>date</th>\n",
       "      <th>link</th>\n",
       "      <th>authors</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>She left her husband. He killed their children...</td>\n",
       "      <td>There Were 2 Mass Shootings In Texas Last Week...</td>\n",
       "      <td>2018-05-26</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/texas-ama...</td>\n",
       "      <td>Melissa Jeltsen</td>\n",
       "      <td>CRIME</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Of course it has a song.</td>\n",
       "      <td>Will Smith Joins Diplo And Nicky Jam For The 2...</td>\n",
       "      <td>2018-05-26</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/will-smit...</td>\n",
       "      <td>Andy McDonald</td>\n",
       "      <td>ENTERTAINMENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The actor and his longtime girlfriend Anna Ebe...</td>\n",
       "      <td>Hugh Grant Marries For The First Time At Age 57</td>\n",
       "      <td>2018-05-26</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/hugh-gran...</td>\n",
       "      <td>Ron Dicker</td>\n",
       "      <td>ENTERTAINMENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The actor gives Dems an ass-kicking for not fi...</td>\n",
       "      <td>Jim Carrey Blasts 'Castrato' Adam Schiff And D...</td>\n",
       "      <td>2018-05-26</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/jim-carre...</td>\n",
       "      <td>Ron Dicker</td>\n",
       "      <td>ENTERTAINMENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The \"Dietland\" actress said using the bags is ...</td>\n",
       "      <td>Julianna Margulies Uses Donald Trump Poop Bags...</td>\n",
       "      <td>2018-05-26</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/julianna-...</td>\n",
       "      <td>Ron Dicker</td>\n",
       "      <td>ENTERTAINMENT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   short_description  \\\n",
       "0  She left her husband. He killed their children...   \n",
       "1                           Of course it has a song.   \n",
       "2  The actor and his longtime girlfriend Anna Ebe...   \n",
       "3  The actor gives Dems an ass-kicking for not fi...   \n",
       "4  The \"Dietland\" actress said using the bags is ...   \n",
       "\n",
       "                                            headline       date  \\\n",
       "0  There Were 2 Mass Shootings In Texas Last Week... 2018-05-26   \n",
       "1  Will Smith Joins Diplo And Nicky Jam For The 2... 2018-05-26   \n",
       "2    Hugh Grant Marries For The First Time At Age 57 2018-05-26   \n",
       "3  Jim Carrey Blasts 'Castrato' Adam Schiff And D... 2018-05-26   \n",
       "4  Julianna Margulies Uses Donald Trump Poop Bags... 2018-05-26   \n",
       "\n",
       "                                                link          authors  \\\n",
       "0  https://www.huffingtonpost.com/entry/texas-ama...  Melissa Jeltsen   \n",
       "1  https://www.huffingtonpost.com/entry/will-smit...    Andy McDonald   \n",
       "2  https://www.huffingtonpost.com/entry/hugh-gran...       Ron Dicker   \n",
       "3  https://www.huffingtonpost.com/entry/jim-carre...       Ron Dicker   \n",
       "4  https://www.huffingtonpost.com/entry/julianna-...       Ron Dicker   \n",
       "\n",
       "        category  \n",
       "0          CRIME  \n",
       "1  ENTERTAINMENT  \n",
       "2  ENTERTAINMENT  \n",
       "3  ENTERTAINMENT  \n",
       "4  ENTERTAINMENT  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_json('News_Category_Dataset.json', lines = True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total categories:  30\n",
      "category\n",
      "ARTS               1509\n",
      "ARTS & CULTURE     1339\n",
      "BLACK VOICES       3858\n",
      "BUSINESS           4254\n",
      "COLLEGE            1144\n",
      "COMEDY             3971\n",
      "CRIME              2893\n",
      "EDUCATION          1004\n",
      "ENTERTAINMENT     14257\n",
      "FIFTY              1401\n",
      "GOOD NEWS          1398\n",
      "GREEN              2622\n",
      "HEALTHY LIVING     6694\n",
      "IMPACT             2602\n",
      "LATINO VOICES      1129\n",
      "MEDIA              2815\n",
      "PARENTS            3955\n",
      "POLITICS          32739\n",
      "QUEER VOICES       4995\n",
      "RELIGION           2556\n",
      "SCIENCE            1381\n",
      "SPORTS             4167\n",
      "STYLE              2254\n",
      "TASTE              2096\n",
      "TECH               1231\n",
      "TRAVEL             2145\n",
      "WEIRD NEWS         2670\n",
      "WOMEN              3490\n",
      "WORLD NEWS         2177\n",
      "WORLDPOST          6243\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#将数据按照字段'category‘分类\n",
    "categories = df.groupby('category')\n",
    "df.category = df.category.map(lambda x:\"WORLDPOST\" if x == \"THE WORLDPOST\" else x)\n",
    "print(\"total categories: \", categories.ngroups)\n",
    "print(categories.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/lancer/anaconda3/envs/keras1/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0                                                                     There Were 2 Mass Shootings In Texas Last Week, But Only 1 On TV She left her husband. He killed their children. Just another day in America.\n",
      "1                                                                                                              Will Smith Joins Diplo And Nicky Jam For The 2018 World Cup's Official Song Of course it has a song.\n",
      "2                                                                           Hugh Grant Marries For The First Time At Age 57 The actor and his longtime girlfriend Anna Eberstein tied the knot in a civil ceremony.\n",
      "3                                                      Jim Carrey Blasts 'Castrato' Adam Schiff And Democrats In New Artwork The actor gives Dems an ass-kicking for not fighting hard enough against Donald Trump.\n",
      "4                                                   Julianna Margulies Uses Donald Trump Poop Bags To Pick Up After Her Dog The \"Dietland\" actress said using the bags is a \"really cathartic, therapeutic moment.\"\n",
      "5    Morgan Freeman 'Devastated' That Sexual Harassment Claims Could Undermine Legacy \"It is not right to equate horrific incidents of sexual assault with misplaced compliments or humor,\" he said in a statement.\n",
      "6                                                                                                                        Donald Trump Is Lovin' New McDonald's Jingle In 'Tonight Show' Bit It's catchy, all right.\n",
      "7                                                                                                                 What To Watch On Amazon Prime That’s New This Week There's a great mini-series joining this week.\n",
      "8                                                                             Mike Myers Reveals He'd 'Like To' Do A Fourth Austin Powers Film Myer's kids may be pushing for a new \"Powers\" film more than anyone.\n",
      "9                                                                                                                   What To Watch On Hulu That’s New This Week You're getting a recent Academy Award-winning movie.\n",
      "Name: text, dtype: object\n",
      "[[87, 95, 260, 917, 2154, 6, 453, 133, 119, 30, 120, 225, 9, 392, 89, 424, 50, 1003, 38, 323, 44, 202, 51, 185, 73, 6, 168], [34, 1516, 2197, 20046, 5, 18729, 5873, 8, 1, 1579, 76, 26510, 703, 926, 4, 758, 12, 31, 3, 926], [5201, 5146, 8954, 8, 1, 69, 59, 19, 463, 7901, 1, 670, 5, 28, 4258, 2641, 3983, 47693, 2789, 1, 8955, 6, 3, 719, 2574], [2198, 9428, 2458, 47694, 2030, 8956, 5, 287, 6, 27, 7603, 1, 670, 642, 3529, 32, 4424, 5701, 8, 23, 738, 343, 350, 112, 55, 20], [36179, 26511, 1605, 55, 20, 6883, 4637, 2, 958, 45, 46, 50, 512, 1, 47695, 820, 67, 627, 1, 4637, 7, 3, 141, 30248, 13758, 435], [3894, 11482, 20047, 10, 267, 861, 450, 86, 3798, 1286, 12, 7, 23, 132, 2, 26512, 3304, 4504, 4, 267, 553, 11, 17539, 13759, 54, 3358, 38, 67, 6, 3, 1396], [55, 20, 7, 14367, 27, 3404, 14368, 6, 9429, 1760, 1141, 56, 12715, 40, 132], [33, 2, 178, 9, 1839, 1223, 1560, 27, 16, 119, 341, 3, 217, 4337, 448, 3503, 16, 119], [735, 11483, 775, 2459, 9181, 21715, 62, 3, 1849, 3799, 3071, 334, 47696, 164, 96, 17, 2416, 8, 3, 27, 3071, 334, 41, 66, 683], [33, 2, 178, 9, 8745, 1560, 27, 16, 119, 262, 258, 3, 485, 2761, 1192, 961, 416]]\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing import sequence\n",
    "from keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "#显示所有列\n",
    "pd.set_option('display.max_columns', None)\n",
    "#显示所有行\n",
    "pd.set_option('display.max_rows', None)\n",
    "#设置value的显示长度为100，默认为50\n",
    "pd.set_option('max_colwidth',None)\n",
    "\n",
    "df['text'] = df.headline + \" \" + df.short_description\n",
    "print(df.text[0:10])\n",
    "#将单词进行标号\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(df.text)\n",
    "X = tokenizer.texts_to_sequences(df.text)\n",
    "df['words'] = X\n",
    "print(X[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    124068.000000\n",
       "mean         26.128422\n",
       "std          14.366390\n",
       "min           5.000000\n",
       "25%          17.000000\n",
       "50%          24.000000\n",
       "75%          32.000000\n",
       "max         248.000000\n",
       "Name: word_length, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#记录每条数据的单词数\n",
    "df['word_length'] = df.words.apply(lambda i:len(i))\n",
    "#清除单词数不足5个的数据条目\n",
    "df = df[df.word_length >= 5]\n",
    "df.word_length.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 145130)\n"
     ]
    }
   ],
   "source": [
    "def word2Frequent(sequences):\n",
    "    word_index = {}\n",
    "    for seq in sequences:\n",
    "        for word in seq:\n",
    "            word_index[word] = word_index.get(word, 0) + 1\n",
    "    return word_index\n",
    "\n",
    "word_index = word2Frequent(df.words)\n",
    "count = 10000\n",
    "#排序取频率在前10000的单词\n",
    "s = [(k, word_index[k]) for k in sorted(word_index, key = word_index.get, reverse = True)]\n",
    "print(s[0])\n",
    "frequent_to_index = {}\n",
    "for i in range(count):\n",
    "    frequent_to_index[s[i][0]] = 9999 - i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将分量进行编号\n",
    "categories = df.groupby('category').size().index.tolist()\n",
    "category_int = {}\n",
    "int_category = {}\n",
    "for i,k in enumerate(categories):\n",
    "    category_int.update({k:i})\n",
    "    int_category.update({i:k})\n",
    "df['c2id'] = df['category'].apply(lambda x:category_int[x])\n",
    "print(df['c2id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 0. 0. ... 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import keras.utils as utils\n",
    "from sklearn.model_selection import train_test_split\n",
    "#将句子分解成单词，然后转换为向量\n",
    "def vectorize_sequence(sequences, dimension = 10000):\n",
    "    results = np.zeros((len(sequences), dimension))\n",
    "    for i in range(len(sequences)):\n",
    "        for word in sequences[i]:\n",
    "            if frequent_to_index.get(word, None) is not None:\n",
    "                pos = frequent_to_index[word]\n",
    "                results[i, pos] += 1.0\n",
    "    return results\n",
    "\n",
    "X = np.array(df.words)\n",
    "X = vectorize_sequence(X)\n",
    "print(X[0])\n",
    "Y = utils.to_categorical(list(df['c2id']))\n",
    "#将数据分为两部分\n",
    "seed = 29\n",
    "x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size = 0.2,\n",
    "                                                 random_state = seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(64, activation = 'relu', input_shape = (10000,)))\n",
    "model.add(layers.Dense(64, activation = 'relu'))\n",
    "model.add(layers.Dense(len(int_categoly), activation = 'softmax'))\n",
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy',metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(x_train, y_train, epochs=20, validation_data=(x_val, y_val),batch_size=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "acc = history.history['acc']\n",
    "val_acc = history.history['val_acc']\n",
    "epochs = range(1, len(loss) + 1)\n",
    "plt.plot(epochschs, acc, 'bo', label = 'Training acc')\n",
    "plt.plot(epochschs, val_acc, 'b', label = 'Validation acc')\n",
    "plt.xlable('Epochs')\n",
    "plt.ylable('acc')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = models.Sequential()\n",
    "model1.add(layers.Dense(64, activation = 'relu', input_shape = (10000,)))\n",
    "model1.add(layers.Dense(64, activation = 'relu'))\n",
    "model1.add(layers.Dense(len(int_categoly), activation = 'softmax'))\n",
    "model1.compile(optimizer='rmsprop', loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "history = model1.fit(x_train, y_train, epochs=4, validation_data=(x_val, y_val),batch_size=512)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python [conda env:keras1] *",
   "language": "python",
   "name": "conda-env-keras1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
